core: Clean up staging API and internals

Cleanly separate metadata/content APIs, rather than defaulting to
raw streams.  This helps most use cases.

Also, drop support for staging content without knowing the total
length.  This complicated the code, and for things like streaming
HTTP, we should be able to figure this out from Content-Length.

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 05041b454361f69fc0235c3311975ea01c102e4d..30cb3e49608d1a1624929b5d13056d10cd6735cf 100644 (file)
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -824,22 +824,17 @@ commit_loose_object_trusted (OstreeRepo        *self,
   return ret;
 }
 
-typedef enum {
-  OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID = (1<<0)
-} OstreeRepoStageFlags;
-
 static gboolean
-stage_object_internal (OstreeRepo         *self,
-                       OstreeRepoStageFlags flags,
-                       OstreeObjectType    objtype,
-                       GInputStream       *input,
-                       guint64             file_object_length,
-                       const char         *expected_checksum,
-                       guchar            **out_csum,
-                       GCancellable       *cancellable,
-                       GError            **error)
+stage_object (OstreeRepo         *self,
+              OstreeObjectType    objtype,
+              const char         *expected_checksum,
+              GInputStream       *input,
+              guint64             file_object_length,
+              guchar            **out_csum,
+              GCancellable       *cancellable,
+              GError            **error)
 {
- gboolean ret = FALSE;
+  gboolean ret = FALSE;
   const char *actual_checksum;
   gboolean do_commit;
   ot_lobj GFileInfo *temp_info = NULL;
@@ -854,6 +849,20 @@ stage_object_internal (OstreeRepo         *self,
   gboolean staged_archive_file = FALSE;
   gboolean temp_file_is_regular;
 
+  g_return_val_if_fail (self->in_transaction, FALSE);
+  
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  g_assert (expected_checksum || out_csum);
+
+  if (expected_checksum)
+    {
+      if (!repo_find_object (self, objtype, expected_checksum, &stored_path,
+                             cancellable, error))
+        goto out;
+    }
+
   if (out_csum)
     {
       checksum = g_checksum_new (G_CHECKSUM_SHA256);
@@ -861,8 +870,7 @@ stage_object_internal (OstreeRepo         *self,
         checksum_input = ostree_checksum_input_stream_new (input, checksum);
     }
 
-  if (objtype == OSTREE_OBJECT_TYPE_FILE
-      && (flags & OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID) > 0)
+  if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
       ot_lobj GInputStream *file_input = NULL;
       ot_lobj GFileInfo *file_info = NULL;
@@ -1038,50 +1046,6 @@ stage_object_internal (OstreeRepo         *self,
   return ret;
 }
 
-static gboolean
-stage_object (OstreeRepo         *self,
-              OstreeRepoStageFlags flags,
-              OstreeObjectType    objtype,
-              GInputStream       *input,
-              guint64             file_object_length,
-              const char         *expected_checksum,
-              guchar            **out_csum,
-              GCancellable       *cancellable,
-              GError            **error)
-{
-  gboolean ret = FALSE;
-  ot_lobj GFile *stored_path = NULL;
-  ot_lfree guchar *ret_csum = NULL;
-
-  g_return_val_if_fail (self->in_transaction, FALSE);
-  
-  if (g_cancellable_set_error_if_cancelled (cancellable, error))
-    return FALSE;
-
-  g_assert (expected_checksum || out_csum);
-
-  if (expected_checksum)
-    {
-      if (!repo_find_object (self, objtype, expected_checksum, &stored_path,
-                             cancellable, error))
-        goto out;
-    }
-
-  if (stored_path == NULL)
-    {
-      if (!stage_object_internal (self, flags, objtype, input,
-                                  file_object_length, expected_checksum,
-                                  out_csum ? &ret_csum : NULL,
-                                  cancellable, error))
-        goto out;
-    }
-
-  ret = TRUE;
-  ot_transfer_out_value(out_csum, &ret_csum);
- out:
-  return ret;
-}
-
 static gboolean
 get_loose_object_dirs (OstreeRepo       *self,
                        GPtrArray       **out_object_dirs,
@@ -1322,26 +1286,56 @@ ostree_repo_abort_transaction (OstreeRepo     *self,
   return ret;
 }
 
-static gboolean
-stage_metadata_object (OstreeRepo         *self,
-                       OstreeObjectType    type,
-                       GVariant           *variant,
-                       guchar            **out_csum,
-                       GCancellable       *cancellable,
-                       GError            **error)
+/**
+ * ostree_repo_stage_metadata:
+ * 
+ * Store the metadata object @variant.  Return the checksum
+ * as @out_csum.
+ *
+ * If @expected_checksum is not %NULL, verify it against the
+ * computed checksum.
+ */
+gboolean
+ostree_repo_stage_metadata (OstreeRepo         *self,
+                            OstreeObjectType    type,
+                            const char         *expected_checksum,
+                            GVariant           *variant,
+                            guchar            **out_csum,
+                            GCancellable       *cancellable,
+                            GError            **error)
 {
-  gboolean ret = FALSE;
   ot_lobj GInputStream *input = NULL;
+  ot_lvariant GVariant *normalized = NULL;
 
-  input = ot_variant_read (variant);
+  normalized = g_variant_get_normal_form (variant);
+  input = ot_variant_read (normalized);
   
-  if (!stage_object (self, 0, type, input, 0, NULL,
-                     out_csum, cancellable, error))
-    goto out;
+  return stage_object (self, type, expected_checksum, input, 0, out_csum,
+                       cancellable, error);
+}
 
-  ret = TRUE;
- out:
-  return ret;
+/**
+ * ostree_repo_stage_metadata_trusted:
+ * 
+ * Store the metadata object @variant; the provided @checksum
+ * is trusted.
+ */
+gboolean
+ostree_repo_stage_metadata_trusted (OstreeRepo         *self,
+                                    OstreeObjectType    type,
+                                    const char         *checksum,
+                                    GVariant           *variant,
+                                    GCancellable       *cancellable,
+                                    GError            **error)
+{
+  ot_lobj GInputStream *input = NULL;
+  ot_lvariant GVariant *normalized = NULL;
+
+  normalized = g_variant_get_normal_form (variant);
+  input = ot_variant_read (normalized);
+  
+  return stage_object (self, type, checksum, input, 0, NULL,
+                       cancellable, error);
 }
 
 static gboolean
@@ -1352,7 +1346,6 @@ stage_directory_meta (OstreeRepo   *self,
                       GCancellable *cancellable,
                       GError      **error)
 {
-  gboolean ret = FALSE;
   ot_lvariant GVariant *dirmeta = NULL;
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
@@ -1360,13 +1353,8 @@ stage_directory_meta (OstreeRepo   *self,
 
   dirmeta = ostree_create_directory_metadata (file_info, xattrs);
   
-  if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_DIR_META, 
-                              dirmeta, out_csum, cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  return ret;
+  return ostree_repo_stage_metadata (self, OSTREE_OBJECT_TYPE_DIR_META, NULL,
+                                     dirmeta, out_csum, cancellable, error);
 }
 
 GFile *
@@ -1384,77 +1372,49 @@ ostree_repo_get_object_path (OstreeRepo  *self,
   return ret;
 }
 
+/**
+ * ostree_repo_stage_content_trusted:
+ *
+ * Store the content object streamed as @object_input, with total
+ * length @length.  The given @checksum will be treated as trusted.
+ *
+ * This function should be used when importing file objects from local
+ * disk, for example.
+ */
 gboolean      
-ostree_repo_stage_object_trusted (OstreeRepo   *self,
-                                  OstreeObjectType objtype,
-                                  const char   *checksum,
-                                  GInputStream     *object_input,
-                                  GCancellable *cancellable,
-                                  GError      **error)
-{
-  int flags = 0;
-  return stage_object (self, flags, objtype,
-                       object_input, 0, checksum, NULL,
+ostree_repo_stage_content_trusted (OstreeRepo       *self,
+                                   const char       *checksum,
+                                   GInputStream     *object_input,
+                                   guint64           length,
+                                   GCancellable     *cancellable,
+                                   GError          **error)
+{
+  return stage_object (self, OSTREE_OBJECT_TYPE_FILE, checksum,
+                       object_input, length, NULL,
                        cancellable, error);
 }
 
+/**
+ * ostree_repo_stage_content:
+ *
+ * Store the content object streamed as @object_input,
+ * with total length @length.  The actual checksum will
+ * be returned as @out_csum.
+ */
 gboolean
-ostree_repo_stage_object (OstreeRepo       *self,
-                          OstreeObjectType  objtype,
-                          const char       *expected_checksum,
-                          GInputStream     *object_input,
-                          GCancellable     *cancellable,
-                          GError          **error)
+ostree_repo_stage_content (OstreeRepo       *self,
+                           const char       *expected_checksum,
+                           GInputStream     *object_input,
+                           guint64           length,
+                           guchar          **out_csum,
+                           GCancellable     *cancellable,
+                           GError          **error)
 {
-  gboolean ret = FALSE;
-  ot_lfree guchar *actual_csum = NULL;
-  
-  if (!stage_object (self, 0, objtype, 
-                     object_input, 0, expected_checksum, &actual_csum,
-                     cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  return ret;
-}
-
-gboolean      
-ostree_repo_stage_file_object_trusted (OstreeRepo       *self,
-                                       const char       *checksum,
-                                       GInputStream     *object_input,
-                                       guint64           length,
-                                       GCancellable     *cancellable,
-                                       GError          **error)
-{
-  int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
-  return stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE,
-                       object_input, length, checksum, NULL,
+  return stage_object (self, OSTREE_OBJECT_TYPE_FILE, expected_checksum,
+                       object_input, length, out_csum,
                        cancellable, error);
 }
 
-gboolean
-ostree_repo_stage_file_object (OstreeRepo       *self,
-                               const char       *expected_checksum,
-                               GInputStream     *object_input,
-                               guint64           length,
-                               GCancellable     *cancellable,
-                               GError          **error)
-{
-  gboolean ret = FALSE;
-  int flags = OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID;
-  ot_lfree guchar *actual_csum = NULL;
-  
-  if (!stage_object (self, flags, OSTREE_OBJECT_TYPE_FILE, 
-                     object_input, length, expected_checksum, &actual_csum,
-                     cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  return ret;
-}
-
 static GVariant *
 create_empty_gvariant_dict (void)
 {
@@ -1658,8 +1618,9 @@ ostree_repo_stage_commit (OstreeRepo *self,
                           ostree_checksum_to_bytes_v (root_contents_checksum),
                           ostree_checksum_to_bytes_v (root_metadata_checksum));
   g_variant_ref_sink (commit);
-  if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_COMMIT,
-                              commit, &commit_csum, cancellable, error))
+  if (!ostree_repo_stage_metadata (self, OSTREE_OBJECT_TYPE_COMMIT, NULL,
+                                   commit, &commit_csum,
+                                   cancellable, error))
     goto out;
 
   ret_commit = ostree_checksum_from_bytes (commit_csum);
@@ -1932,9 +1893,8 @@ stage_directory_to_mtree_internal (OstreeRepo           *self,
                                                               &file_object_input, &file_obj_length,
                                                               cancellable, error))
                         goto out;
-                      if (!stage_object (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID,
-                                         OSTREE_OBJECT_TYPE_FILE, file_object_input, file_obj_length,
-                                         NULL, &child_file_csum, cancellable, error))
+                      if (!ostree_repo_stage_content (self, NULL, file_object_input, file_obj_length,
+                                                      &child_file_csum, cancellable, error))
                         goto out;
 
                       g_free (tmp_checksum);
@@ -2041,9 +2001,9 @@ ostree_repo_stage_mtree (OstreeRepo           *self,
                                                          dir_contents_checksums,
                                                          dir_metadata_checksums);
       
-      if (!stage_metadata_object (self, OSTREE_OBJECT_TYPE_DIR_TREE,
-                                  serialized_tree, &contents_csum,
-                                  cancellable, error))
+      if (!ostree_repo_stage_metadata (self, OSTREE_OBJECT_TYPE_DIR_TREE, NULL,
+                                       serialized_tree, &contents_csum,
+                                       cancellable, error))
         goto out;
       ret_contents_checksum = ostree_checksum_from_bytes (contents_csum);
     }
@@ -2140,9 +2100,8 @@ import_libarchive_entry_file (OstreeRepo           *self,
                                           &file_object_input, &length, cancellable, error))
     goto out;
   
-  if (!stage_object (self, OSTREE_REPO_STAGE_FLAGS_LENGTH_VALID, OSTREE_OBJECT_TYPE_FILE,
-                     file_object_input, length, NULL, out_csum,
-                     cancellable, error))
+  if (!ostree_repo_stage_content (self, NULL, file_object_input, length, out_csum,
+                                  cancellable, error))
     goto out;
 
   ret = TRUE;

diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 151ea3afbc68426a289e23e6bbc01211611e5dc0..2f31bef0ae3b8db88931d59dfe1c123869a65051 100644 (file)
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -94,33 +94,35 @@ gboolean      ostree_repo_has_object (OstreeRepo           *self,
                                       GCancellable         *cancellable,
                                       GError              **error);
 
-gboolean      ostree_repo_stage_object (OstreeRepo       *self,
-                                        OstreeObjectType  objtype,
-                                        const char       *expected_checksum,
-                                        GInputStream     *content,
-                                        GCancellable     *cancellable,
-                                        GError          **error);
-
-gboolean      ostree_repo_stage_file_object (OstreeRepo       *self,
-                                             const char       *expected_checksum,
-                                             GInputStream     *content,
-                                             guint64           content_length,
-                                             GCancellable     *cancellable,
-                                             GError          **error);
-
-gboolean      ostree_repo_stage_object_trusted (OstreeRepo   *self,
-                                                OstreeObjectType objtype,
-                                                const char   *checksum,
-                                                GInputStream     *content,
-                                                GCancellable *cancellable,
-                                                GError      **error);
-
-gboolean      ostree_repo_stage_file_object_trusted (OstreeRepo   *self,
-                                                     const char   *checksum,
-                                                     GInputStream     *content,
-                                                     guint64           content_length,
-                                                     GCancellable *cancellable,
-                                                     GError      **error);
+gboolean      ostree_repo_stage_metadata (OstreeRepo        *self,
+                                          OstreeObjectType   objtype,
+                                          const char        *expected_checksum,
+                                          GVariant          *object,
+                                          guchar           **out_csum,
+                                          GCancellable      *cancellable,
+                                          GError           **error);
+
+gboolean      ostree_repo_stage_content (OstreeRepo       *self,
+                                         const char       *expected_checksum,
+                                         GInputStream     *content,
+                                         guint64           content_length,
+                                         guchar          **out_csum,
+                                         GCancellable     *cancellable,
+                                         GError          **error);
+
+gboolean      ostree_repo_stage_metadata_trusted (OstreeRepo        *self,
+                                                  OstreeObjectType   objtype,
+                                                  const char        *checksum,
+                                                  GVariant          *object,
+                                                  GCancellable      *cancellable,
+                                                  GError           **error);
+
+gboolean      ostree_repo_stage_content_trusted (OstreeRepo       *self,
+                                                 const char       *checksum,
+                                                 GInputStream     *content,
+                                                 guint64           content_length,
+                                                 GCancellable     *cancellable,
+                                                 GError          **error);
 
 gboolean      ostree_repo_resolve_rev (OstreeRepo  *self,
                                        const char  *rev,

diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index d1934b474304cdaa8354c3849941b4dbf0e743d8..7122f5b03ab4db2533f868fa04d34d0ff310279f 100644 (file)
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -359,7 +359,6 @@ fetch_and_store_metadata (OtPullData          *pull_data,
   gboolean is_stored;
   ot_lvariant GVariant *ret_variant = NULL;
   ot_lobj GFile *temp_path = NULL;
-  ot_lobj GInputStream *input = NULL;
   ot_lvariant GVariant *metadata = NULL;
 
   g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype));
@@ -370,15 +369,32 @@ fetch_and_store_metadata (OtPullData          *pull_data,
       
   if (!is_stored)
     {
+      ot_lvariant GVariant *tmp_metadata = NULL;
+      const GVariantType *vtype;
+  
       if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
         goto out;
-      
-      input = (GInputStream*)g_file_read (temp_path, cancellable, error);
-      if (!input)
-        goto out;
 
-      if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input,
-                                     cancellable, error))
+      switch (objtype)
+        {
+        case OSTREE_OBJECT_TYPE_DIR_TREE:
+          vtype = OSTREE_TREE_GVARIANT_FORMAT;
+          break;
+        case OSTREE_OBJECT_TYPE_DIR_META:
+          vtype = OSTREE_DIRMETA_GVARIANT_FORMAT;
+          break;
+        case OSTREE_OBJECT_TYPE_COMMIT:
+          vtype = OSTREE_COMMIT_GVARIANT_FORMAT;
+          break;
+        default:
+          g_assert_not_reached ();
+        }
+
+      if (!ot_util_variant_map (temp_path, vtype, FALSE, &tmp_metadata, error))
+        goto out;
+      
+      if (!ostree_repo_stage_metadata (pull_data->repo, objtype, checksum, tmp_metadata, NULL,
+                                       cancellable, error))
         goto out;
     }
 
@@ -664,9 +680,9 @@ content_fetch_on_checksum_complete (GObject        *object,
       goto out;
     }
 
-  if (!ostree_repo_stage_file_object_trusted (data->pull_data->repo, checksum,
-                                              file_object_input, length,
-                                              cancellable, error))
+  if (!ostree_repo_stage_content_trusted (data->pull_data->repo, checksum,
+                                          file_object_input, length,
+                                          cancellable, error))
     goto out;
 
  out:

diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index 7d6e05d0249b78285ce4663698fd5374713b92e7..8b7304db3b5e7cc7af2c578dc9768b00fc2fcc2c 100644 (file)
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -45,17 +45,16 @@ import_one_object (OtLocalCloneData *data,
                    GError        **error)
 {
   gboolean ret = FALSE;
-  ot_lobj GFileInfo *file_info = NULL;
   ot_lobj GFile *content_path = NULL;
   ot_lobj GFileInfo *archive_info = NULL;
-  ot_lvariant GVariant *metadata = NULL;
-  ot_lvariant GVariant *xattrs = NULL;
-  ot_lobj GInputStream *input = NULL;
 
   if (objtype == OSTREE_OBJECT_TYPE_FILE)
     {
-      ot_lobj GInputStream *file_object = NULL;
       guint64 length;
+      ot_lobj GInputStream *file_object = NULL;
+      ot_lobj GInputStream *input = NULL;
+      ot_lobj GFileInfo *file_info = NULL;
+      ot_lvariant GVariant *xattrs = NULL;
 
       if (!ostree_repo_load_file (data->src_repo, checksum,
                                   &input, &file_info, &xattrs,
@@ -67,22 +66,21 @@ import_one_object (OtLocalCloneData *data,
                                               cancellable, error))
         goto out;
 
-      if (!ostree_repo_stage_file_object_trusted (data->dest_repo, checksum,
-                                                  file_object, length,
-                                                  cancellable, error))
+      if (!ostree_repo_stage_content_trusted (data->dest_repo, checksum,
+                                              file_object, length,
+                                              cancellable, error))
         goto out;
     }
   else
     {
+      ot_lvariant GVariant *metadata = NULL;
+
       if (!ostree_repo_load_variant (data->src_repo, objtype, checksum, &metadata,
                                      error))
         goto out;
 
-      input = ot_variant_read (metadata);
-
-      if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype,
-                                             checksum, input,
-                                             cancellable, error))
+      if (!ostree_repo_stage_metadata_trusted (data->dest_repo, objtype, checksum, metadata,
+                                               cancellable, error))
         goto out;
     }